package com.xhji.spider.webmagic.processor;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.xhji.spider.entity.PageDetail;
import com.xhji.spider.entity.PageDetailRule;
import com.xhji.spider.utils.StringUtil;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

public class PageDetailProcessor implements PageProcessor {

    private Site site;
    private PageDetailRule rule;
    private Set<String> excludeUrl = new HashSet<>();

    private PageDetailProcessor(){}

    public PageDetailProcessor(PageDetailRule rule) {
        this.site = getSite(rule);
        this.rule = rule;
    }

    public PageDetailProcessor(PageDetailRule rule, Set<String> set) {
        this.site = getSite(rule);
        this.rule = rule;
        this.excludeUrl = set;
    }

    @Override
    public void process(Page page) {
        //如果是辅助页url  那么从这个也没找到所以的目标页url和其他辅助页url
        List<String> targetList = page.getHtml().links().regex(rule.getTargetUrlRegex()).all().stream().map(StringUtil::encodeChinese).collect(Collectors.toList());
        List<String> helpList = page.getHtml().links().regex(rule.getHelpUrlRegex()).all().stream().map(StringUtil::encodeChinese).collect(Collectors.toList());
        if (page.getUrl().regex(rule.getHelpUrlRegex()).match()){
            if (StringUtils.hasText(rule.getTargetUrlModuleRegex())){
                page.addTargetRequests(page.getHtml().xpath(rule.getTargetUrlModuleRegex()).links().regex(rule.getTargetUrlRegex()).all().stream().map(StringUtil::encodeChinese).collect(Collectors.toList()));
            }else {
                page.addTargetRequests(targetList);
            }
            page.addTargetRequests(helpList);
            page.setSkip(true);
            return;
        }else {
            page.addTargetRequests(targetList);
            page.addTargetRequests(helpList);
        }
        //已经下载过的页面不做处理
        if (excludeUrl.contains(page.getRequest().getUrl())){
            page.setSkip(true);
            return;
        }
        Html html = page.getHtml();
        PageDetail pageDetail = new PageDetail();
        pageDetail.setUrl(page.getRequest().getUrl());
        pageDetail.setDomain(rule.getDomain());
        pageDetail.setRuleId(rule.getId());
//        pageDetail.setHtml(html.getDocument().html());
//        System.out.println(html.getDocument().html());
        pageDetail.setCreateDate(LocalDateTime.now());
        pageDetail.setIsDownload(true);
        pageDetail.setIsSuccess(false);
        if (StringUtils.hasText(rule.getAuthorRegex())){
            try {
                pageDetail.setAuthor(html.xpath(rule.getAuthorRegex()).toString());
            } catch (Exception e) {
                pageDetail.setAuthor(html.regex(rule.getAuthorRegex()).toString());
            }
        }
        if (StringUtils.hasText(rule.getCatalogRegex())){
            pageDetail.setCategory(html.xpath(rule.getCatalogRegex()).toString());
        }
        if (StringUtils.hasText(rule.getContentRegex())){
            pageDetail.setContent(html.xpath(rule.getContentRegex()).toString());
        }
        if (StringUtils.hasText(rule.getDateRegex())){
            pageDetail.setDate(html.xpath(rule.getDateRegex()).toString());
        }
        if (StringUtils.hasText(rule.getDescriptionRegex())){
            pageDetail.setDescription(html.xpath(rule.getDescriptionRegex()).toString());
        }
        if (StringUtils.hasText(rule.getTagRegex())){
            pageDetail.setTags(html.xpath(rule.getTagRegex()).toString());
        }
        if (StringUtils.hasText(rule.getImageRegex())){
            List<String> list = html.xpath(rule.getImageRegex()).all();
            pageDetail.setImageNum(list.size());
            pageDetail.setImages(String.join(",", list));
        }
        if (StringUtils.hasText(rule.getTitleRegex())){
            pageDetail.setTitle(html.xpath(rule.getTitleRegex()).toString());
        }else {
            pageDetail.setTitle(page.getHtml().getDocument().title());
        }
        if (!StringUtils.hasText(pageDetail.getTitle())){
            page.setSkip(true);
        }else {
            page.putField("pageDetail", pageDetail);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    private Site getSite(PageDetailRule rule){
        Site site = Site.me().setDomain(rule.getDomain());
        site.addHeader("Refer", rule.getDomain());
        if (StringUtils.hasText(rule.getUserAgent())){
            site.setUserAgent(rule.getUserAgent());
        }
        if (StringUtils.hasText(rule.getCookies())){
            try {
                String[] split = rule.getCookies().split("&");
                for (String s : split) {
                    String[] split1 = s.split("=");
                    site.addCookie(split1[0], split1[1]);
                }
            } catch (Exception e) {
                System.out.println("cookies格式错误, ruleID：" + rule.getId());
            }
        }
        if (StringUtils.hasText(rule.getHeader())){
            try {
                JSONObject jsonObject = JSON.parseObject(rule.getHeader());
                jsonObject.forEach((key, value) -> site.addHeader(key, String.valueOf(value)));
            } catch (Exception e) {
                System.out.println("header格式错误, ruleID：" + rule.getId());
            }
        }
        if (rule.getRetryTimes() != null){
            site.setRetryTimes(rule.getRetryTimes());
        }
        if (rule.getSleepTime() != null){
            site.setSleepTime(rule.getSleepTime());
        }
        return site;
    }
}
